{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "e76f66b4",
   "metadata": {},
   "outputs": [],
   "source": [
    "from datasets import *"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dbf813f4",
   "metadata": {},
   "source": [
    "# Datasets基本使用"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2a0726f0",
   "metadata": {},
   "source": [
    "## 加载"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "589257f7",
   "metadata": {},
   "source": [
    "### 加载在线数据集"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "50bbe90c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['title', 'content'],\n",
       "        num_rows: 5850\n",
       "    })\n",
       "    validation: Dataset({\n",
       "        features: ['title', 'content'],\n",
       "        num_rows: 1679\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\"需要科学上网\"\"\"\n",
    "datasets=load_dataset(\"madao33/new-title-chinese\",cache_dir=\"download_datasets/new-title-chinese\")\n",
    "datasets"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "175d4ba4",
   "metadata": {},
   "source": [
    "### 加载数据集合集中某一项任务"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "bbd13a8c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b1a62628e28743d685c561dc1b1dd0e1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading data:   0%|          | 0.00/4.12M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "37cca08361514bc9a21b4192c75c2762",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating train split:   0%|          | 0/9427 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7de4c76a9c70480eaff6daaefaef2496",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating validation split:   0%|          | 0/3270 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7a24f9b7604e4a53849933251ec780e8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating test split:   0%|          | 0/3245 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "boolq_dataset=load_dataset(\"super_glue\",\"boolq\",cache_dir=\"download_datasets/super_glue_boolq\",trust_remote_code=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "4a3e7d92",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['question', 'passage', 'idx', 'label'],\n",
       "        num_rows: 9427\n",
       "    })\n",
       "    validation: Dataset({\n",
       "        features: ['question', 'passage', 'idx', 'label'],\n",
       "        num_rows: 3270\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['question', 'passage', 'idx', 'label'],\n",
       "        num_rows: 3245\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "boolq_dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4eb5b4ea",
   "metadata": {},
   "source": [
    "### !从下载后的文件加载!\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "dbbad198",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using the latest cached version of the module from C:\\Users\\Administrator\\.cache\\huggingface\\modules\\datasets_modules\\datasets\\super_glue\\bb9675f958ebfee0d5d6dc5476fafe38c79123727a7258d515c450873dbdbbed (last modified on Wed Apr 30 14:30:55 2025) since it couldn't be found locally at super_glue, or remotely on the Hugging Face Hub.\n"
     ]
    }
   ],
   "source": [
    "boolq_dataset_loaded=load_dataset(\"super_glue\",\"boolq\",cache_dir=\"download_datasets/super_glue_boolq\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "529e256c",
   "metadata": {},
   "source": [
    "### 按照数据集划分进行加载"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "93cf8d7c",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using the latest cached version of the dataset since madao33/new-title-chinese couldn't be found on the Hugging Face Hub\n",
      "Found the latest cached dataset configuration 'default' at download_datasets\\new-title-chinese\\madao33___new-title-chinese\\default\\0.0.0\\be61f6e55257d64aa16e6a5c09ef9451e3f24c40 (last modified on Wed Apr 30 14:13:08 2025).\n"
     ]
    }
   ],
   "source": [
    "dataset=load_dataset(\"madao33/new-title-chinese\",cache_dir=\"download_datasets/new-title-chinese\",split=\"train\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "ee17ec61",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['title', 'content'],\n",
       "    num_rows: 5850\n",
       "})"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "742b6fc4",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using the latest cached version of the dataset since madao33/new-title-chinese couldn't be found on the Hugging Face Hub\n",
      "Found the latest cached dataset configuration 'default' at download_datasets\\new-title-chinese\\madao33___new-title-chinese\\default\\0.0.0\\be61f6e55257d64aa16e6a5c09ef9451e3f24c40 (last modified on Wed Apr 30 14:13:08 2025).\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['title', 'content'],\n",
       "    num_rows: 90\n",
       "})"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 加载第10条到第100条\n",
    "dataset=load_dataset(\"madao33/new-title-chinese\",cache_dir=\"download_datasets/new-title-chinese\",split=\"train[10:100]\")\n",
    "dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "a425c28b",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using the latest cached version of the dataset since madao33/new-title-chinese couldn't be found on the Hugging Face Hub\n",
      "Found the latest cached dataset configuration 'default' at download_datasets\\new-title-chinese\\madao33___new-title-chinese\\default\\0.0.0\\be61f6e55257d64aa16e6a5c09ef9451e3f24c40 (last modified on Wed Apr 30 14:13:08 2025).\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['title', 'content'],\n",
       "    num_rows: 2925\n",
       "})"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 加载训练集的前50%【注意：只能是百分数，不能是小数，否则会报错】\n",
    "dataset=load_dataset(\"madao33/new-title-chinese\",cache_dir=\"download_datasets/new-title-chinese\",split=\"train[:50%]\")\n",
    "dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "44d0f336",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using the latest cached version of the dataset since madao33/new-title-chinese couldn't be found on the Hugging Face Hub\n",
      "Found the latest cached dataset configuration 'default' at download_datasets\\new-title-chinese\\madao33___new-title-chinese\\default\\0.0.0\\be61f6e55257d64aa16e6a5c09ef9451e3f24c40 (last modified on Wed Apr 30 14:13:08 2025).\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[Dataset({\n",
       "     features: ['title', 'content'],\n",
       "     num_rows: 2925\n",
       " }),\n",
       " Dataset({\n",
       "     features: ['title', 'content'],\n",
       "     num_rows: 2925\n",
       " })]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 加载训练集的前50%，后50%\n",
    "dataset=load_dataset(\"madao33/new-title-chinese\",cache_dir=\"download_datasets/new-title-chinese\",split=[\"train[:50%]\",\"train[50%:]\"])\n",
    "dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4c90ed44",
   "metadata": {},
   "source": [
    "## 查看数据集"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "d14dbd63",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using the latest cached version of the dataset since madao33/new-title-chinese couldn't be found on the Hugging Face Hub\n",
      "Found the latest cached dataset configuration 'default' at download_datasets\\new-title-chinese\\madao33___new-title-chinese\\default\\0.0.0\\be61f6e55257d64aa16e6a5c09ef9451e3f24c40 (last modified on Wed Apr 30 14:13:08 2025).\n"
     ]
    }
   ],
   "source": [
    "datasets=load_dataset(\"madao33/new-title-chinese\",cache_dir=\"download_datasets/new-title-chinese\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "d57aa013",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'title': ['望海楼美国打“台湾牌”是危险的赌博', '大力推进高校治理能力建设'],\n",
       " 'content': ['近期，美国国会众院通过法案，重申美国对台湾的承诺。对此，中国外交部发言人表示，有关法案严重违反一个中国原则和中美三个联合公报规定，粗暴干涉中国内政，中方对此坚决反对并已向美方提出严正交涉。\\n事实上，中国高度关注美国国内打“台湾牌”、挑战一中原则的危险动向。近年来，作为“亲台”势力大本营的美国国会动作不断，先后通过“与台湾交往法”“亚洲再保证倡议法”等一系列“挺台”法案，“2019财年国防授权法案”也多处触及台湾问题。今年3月，美参院亲台议员再抛“台湾保证法”草案。众院议员继而在4月提出众院版的草案并在近期通过。上述法案的核心目标是强化美台关系，并将台作为美“印太战略”的重要伙伴。同时，“亲台”议员还有意制造事端。今年2月，5名共和党参议员致信众议院议长，促其邀请台湾地区领导人在国会上发表讲话。这一动议显然有悖于美国与台湾的非官方关系，其用心是实质性改变美台关系定位。\\n上述动向出现并非偶然。在中美建交40周年之际，两国关系摩擦加剧，所谓“中国威胁论”再次沉渣泛起。美国对华认知出现严重偏差，对华政策中负面因素上升，保守人士甚至成立了“当前中国威胁委员会”。在此背景下，美国将台海关系作为战略抓手，通过打“台湾牌”在双边关系中增加筹码。特朗普就任后，国会对总统外交政策的约束力和塑造力加强。其实国会推动通过涉台法案对行政部门不具约束力，美政府在2018年并未提升美台官员互访级别，美军舰也没有“访问”台湾港口，保持着某种克制。但从美总统签署国会通过的法案可以看出，国会对外交产生了影响。立法也为政府对台政策提供更大空间。\\n然而，美国需要认真衡量打“台湾牌”成本。首先是美国应对危机的代价。美方官员和学者已明确发出警告，美国卷入台湾问题得不偿失。美国学者曾在媒体发文指出，如果台海爆发危机，美国可能需要“援助”台湾，进而导致新的冷战乃至与中国大陆的冲突。但如果美国让台湾自己面对，则有损美国的信誉，影响美盟友对同盟关系的支持。其次是对中美关系的危害。历史证明，中美合则两利、斗则两伤。中美关系是当今世界最重要的双边关系之一，保持中美关系的稳定发展，不仅符合两国和两国人民的根本利益，也是国际社会的普遍期待。美国蓄意挑战台湾问题的底线，加剧中美关系的复杂性和不确定性，损害两国在重要领域合作，损人又害己。\\n美国打“台湾牌”是一场危险的赌博。台湾问题是中国核心利益，中国政府和人民决不会对此坐视不理。中国敦促美方恪守一个中国原则和中美三个联合公报规定，阻止美国会审议推进有关法案，妥善处理涉台问题。美国悬崖勒马，才是明智之举。\\n（作者系中国国际问题研究院国际战略研究所副所长）',\n",
       "  '在推进“双一流”高校建设进程中，我们要紧紧围绕为党育人、为国育才，找准问题、破解难题，以一流意识和担当精神，大力推进高校的治理能力建设。\\n增强政治引领力。坚持党对高校工作的全面领导，始终把政治建设摆在首位，增强校党委的政治领导力，全面推进党的建设各项工作。落实立德树人根本任务，把培养社会主义建设者和接班人放在中心位置。紧紧抓住思想政治工作这条生命线，全面加强师生思想政治工作，推进“三全育人”综合改革，将思想政治工作贯穿学校教育管理服务全过程，努力让学生成为德才兼备、全面发展的人才。\\n提升人才聚集力。人才是创新的核心要素，创新驱动本质上是人才驱动。要坚持引育并举，建立绿色通道，探索知名专家举荐制，完善“一事一议”支持机制。在大力支持自然科学人才队伍建设的同时，实施哲学社会科学人才工程。立足实际，在条件成熟的学院探索“一院一策”改革。创新科研组织形式，为人才成长创设空间，建设更加崇尚学术、更加追求卓越、更加关爱学生、更加担当有为的学术共同体。\\n培养学生竞争力。遵循学生成长成才的规律培育人才，着力培养具有国际竞争力的拔尖创新人才和各类专门人才，使优势学科、优秀教师、优质资源、优良环境围绕立德树人的根本任务配置。淘汰“水课”，打造“金课”，全力打造世界一流本科教育。深入推进研究生教育综合改革，加强事关国家重大战略的高精尖急缺人才培养，建设具有国际竞争力的研究生教育。\\n激发科技创新力。在国家急需发展的领域挑大梁，就要更加聚焦科技前沿和国家需求，狠抓平台建设，包括加快牵头“武汉光源”建设步伐，积极参与国家实验室建设，建立校级大型科研仪器设备共享平台。关键核心技术领域“卡脖子”问题，归根结底是基础科学研究薄弱。要加大基础研究的支持力度，推进理论、技术和方法创新，鼓励支持重大原创和颠覆性技术创新，催生一批高水平、原创性研究成果。\\n发展社会服务力。在贡献和服务中体现价值，推动合作共建、多元投入的格局，大力推进政产学研用结合，强化科技成果转移转化及产业化。探索校城融合发展、校地联动发展的新模式，深度融入地方创新发展网络，为地方经济社会发展提供人才支撑，不断拓展和优化社会服务网络。\\n涵育文化软实力。加快体制机制改革，优化学校、学部、学院三级评审机制，充分发挥优秀学者特别是德才兼备的年轻学者在学术治理中的重要作用。牢固树立一流意识、紧紧围绕一流目标、认真执行一流标准，让成就一流事业成为普遍追求和行动自觉。培育具有强大凝聚力的大学文化，营造积极团结、向上向善、干事创业的氛围，让大学成为吸引和留住一大批优秀人才建功立业的沃土，让敢干事、肯干事、能干事的人有更多的荣誉感和获得感。\\n建设中国特色、世界一流大学不是等得来、喊得来的，而是脚踏实地拼出来、干出来的。对标一流，深化改革，坚持按章程办学，构建以一流质量标准为核心的制度规范体系，扎实推进学校综合改革，探索更具活力、更富效率的管理体制和运行机制，我们就一定能构建起具有中国特色的现代大学治理体系，进一步提升管理服务水平和工作效能。\\n（作者系武汉大学校长）']}"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "datasets[\"train\"][:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "8387ee0e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'title': '望海楼美国打“台湾牌”是危险的赌博',\n",
       " 'content': '近期，美国国会众院通过法案，重申美国对台湾的承诺。对此，中国外交部发言人表示，有关法案严重违反一个中国原则和中美三个联合公报规定，粗暴干涉中国内政，中方对此坚决反对并已向美方提出严正交涉。\\n事实上，中国高度关注美国国内打“台湾牌”、挑战一中原则的危险动向。近年来，作为“亲台”势力大本营的美国国会动作不断，先后通过“与台湾交往法”“亚洲再保证倡议法”等一系列“挺台”法案，“2019财年国防授权法案”也多处触及台湾问题。今年3月，美参院亲台议员再抛“台湾保证法”草案。众院议员继而在4月提出众院版的草案并在近期通过。上述法案的核心目标是强化美台关系，并将台作为美“印太战略”的重要伙伴。同时，“亲台”议员还有意制造事端。今年2月，5名共和党参议员致信众议院议长，促其邀请台湾地区领导人在国会上发表讲话。这一动议显然有悖于美国与台湾的非官方关系，其用心是实质性改变美台关系定位。\\n上述动向出现并非偶然。在中美建交40周年之际，两国关系摩擦加剧，所谓“中国威胁论”再次沉渣泛起。美国对华认知出现严重偏差，对华政策中负面因素上升，保守人士甚至成立了“当前中国威胁委员会”。在此背景下，美国将台海关系作为战略抓手，通过打“台湾牌”在双边关系中增加筹码。特朗普就任后，国会对总统外交政策的约束力和塑造力加强。其实国会推动通过涉台法案对行政部门不具约束力，美政府在2018年并未提升美台官员互访级别，美军舰也没有“访问”台湾港口，保持着某种克制。但从美总统签署国会通过的法案可以看出，国会对外交产生了影响。立法也为政府对台政策提供更大空间。\\n然而，美国需要认真衡量打“台湾牌”成本。首先是美国应对危机的代价。美方官员和学者已明确发出警告，美国卷入台湾问题得不偿失。美国学者曾在媒体发文指出，如果台海爆发危机，美国可能需要“援助”台湾，进而导致新的冷战乃至与中国大陆的冲突。但如果美国让台湾自己面对，则有损美国的信誉，影响美盟友对同盟关系的支持。其次是对中美关系的危害。历史证明，中美合则两利、斗则两伤。中美关系是当今世界最重要的双边关系之一，保持中美关系的稳定发展，不仅符合两国和两国人民的根本利益，也是国际社会的普遍期待。美国蓄意挑战台湾问题的底线，加剧中美关系的复杂性和不确定性，损害两国在重要领域合作，损人又害己。\\n美国打“台湾牌”是一场危险的赌博。台湾问题是中国核心利益，中国政府和人民决不会对此坐视不理。中国敦促美方恪守一个中国原则和中美三个联合公报规定，阻止美国会审议推进有关法案，妥善处理涉台问题。美国悬崖勒马，才是明智之举。\\n（作者系中国国际问题研究院国际战略研究所副所长）'}"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "datasets[\"train\"][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "5e678af2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['望海楼美国打“台湾牌”是危险的赌博',\n",
       " '大力推进高校治理能力建设',\n",
       " '坚持事业为上选贤任能',\n",
       " '“大朋友”的话儿记心头',\n",
       " '用好可持续发展这把“金钥匙”']"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "datasets[\"train\"][\"title\"][:5]#批处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "bacc1f26",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'title': Value(dtype='string', id=None),\n",
       " 'content': Value(dtype='string', id=None)}"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "datasets[\"train\"].features"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "77e17a29",
   "metadata": {},
   "source": [
    "## 数据集划分"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "90a63a2d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['title', 'content'],\n",
       "        num_rows: 5265\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['title', 'content'],\n",
       "        num_rows: 585\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset=datasets[\"train\"]\n",
    "dataset.train_test_split(test_size=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "67163f48",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['question', 'passage', 'idx', 'label'],\n",
       "        num_rows: 8484\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['question', 'passage', 'idx', 'label'],\n",
       "        num_rows: 943\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset=boolq_dataset[\"train\"]\n",
    "\"\"\"stratify_by_column=\"label\"：根据 label 列的类别分布进行 分层抽样，确保新训练集和测试集的标签比例与原数据集一致\"\"\"\n",
    "'''根据标签使得样本均衡'''\n",
    "dataset.train_test_split(test_size=0.1,stratify_by_column=\"label\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8afe7ce6",
   "metadata": {},
   "source": [
    "## 数据选取和过滤"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "bc56d02e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['title', 'content'],\n",
       "    num_rows: 1\n",
       "})"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#选取\n",
    "datasets[\"train\"].select([0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "77514c62",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['title', 'content'],\n",
       "    num_rows: 2\n",
       "})"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#选取\n",
    "datasets[\"train\"].select([0,1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0a2191da",
   "metadata": {},
   "outputs": [],
   "source": [
    "#过滤:筛选出含有中国标题的新闻\n",
    "data_filter=datasets[\"train\"].filter(lambda example:\"中国\" in example[\"title\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "088869e1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['聚焦两会，世界探寻中国成功秘诀',\n",
       " '望海楼中国经济的信心来自哪里',\n",
       " '“中国奇迹”助力世界减贫跑出加速度',\n",
       " '和音瞩目历史交汇点上的中国',\n",
       " '中国风采感染世界']"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_filter[\"title\"][:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bd8fbba1",
   "metadata": {},
   "source": [
    "## 数据映射"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "55f25824",
   "metadata": {},
   "outputs": [],
   "source": [
    "def add_prefix(example):\n",
    "    example[\"title\"]=\"prefix: \"+example[\"title\"]\n",
    "    return example\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "896e889d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['prefix: 望海楼美国打“台湾牌”是危险的赌博',\n",
       " 'prefix: 大力推进高校治理能力建设',\n",
       " 'prefix: 坚持事业为上选贤任能',\n",
       " 'prefix: “大朋友”的话儿记心头',\n",
       " 'prefix: 用好可持续发展这把“金钥匙”',\n",
       " 'prefix: 跨越雄关，我们走在大路上',\n",
       " 'prefix: 脱贫奇迹彰显政治优势',\n",
       " 'prefix: 拱卫亿万人共同的绿色梦想',\n",
       " 'prefix: 为党育人、为国育才',\n",
       " 'prefix: 净化网络语言']"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prefix_dataset=datasets.map(add_prefix)\n",
    "prefix_dataset[\"train\"]['title'][:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "98b27ed8",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import AutoTokenizer\n",
    "tokenizer = AutoTokenizer.from_pretrained(r\"E:\\HuggFace_model\\bert-base-chinese\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f4f8af44",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "用于将原始文本数据转换为BERT模型训练所需的输入格式，主要应用于序列生成任务（如文本摘要、标题生成等）\n",
    "\"\"\"\n",
    "def preprocess_function(example):\n",
    "    # 对\"content\"字段编码为模型输入（如新闻正文）\n",
    "    model_inputs = tokenizer(example[\"content\"], max_length=512, truncation=True)\n",
    "    \n",
    "    # 对\"title\"字段编码为标签（如新闻标题）\n",
    "    labels = tokenizer(example[\"title\"], max_length=32, truncation=True)\n",
    "    \n",
    "    # 将标题的token IDs作为模型的标签\n",
    "    model_inputs[\"labels\"] = labels[\"input_ids\"]\n",
    "    return model_inputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "6f55a817",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c634f5cb940f4b1f966bbc701f0405a6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/5850 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: total: 24.3 s\n",
      "Wall time: 24.4 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "processed_datasets=datasets.map(preprocess_function)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "d9091f0e",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"多线程处理\"\"\"\n",
    "def preprocess_function1(example,tokenizer=tokenizer):\n",
    "    # 对\"content\"字段编码为模型输入（如新闻正文）\n",
    "    model_inputs = tokenizer(example[\"content\"], max_length=512, truncation=True)\n",
    "    \n",
    "    # 对\"title\"字段编码为标签（如新闻标题）\n",
    "    labels = tokenizer(example[\"title\"], max_length=32, truncation=True)\n",
    "    \n",
    "    # 将标题的token IDs作为模型的标签\n",
    "    model_inputs[\"labels\"] = labels[\"input_ids\"]\n",
    "    return model_inputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "f1999292",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: total: 46.9 ms\n",
      "Wall time: 46.9 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# 更快\n",
    "processed_datasets=datasets.map(preprocess_function1,num_proc=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "efeb7623",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['title', 'content', 'input_ids', 'token_type_ids', 'attention_mask', 'labels'],\n",
       "        num_rows: 5850\n",
       "    })\n",
       "    validation: Dataset({\n",
       "        features: ['title', 'content', 'input_ids', 'token_type_ids', 'attention_mask', 'labels'],\n",
       "        num_rows: 1679\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "processed_datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "d60b1d4f",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"去掉数据集原始字段，留下处理后的字段\"\"\"\n",
    "# batched=True:对数据批量处理\n",
    "processed_datasets=datasets.map(preprocess_function,batched=True,remove_columns=datasets[\"train\"].column_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "6a101614",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['input_ids', 'token_type_ids', 'attention_mask', 'labels'],\n",
       "        num_rows: 5850\n",
       "    })\n",
       "    validation: Dataset({\n",
       "        features: ['input_ids', 'token_type_ids', 'attention_mask', 'labels'],\n",
       "        num_rows: 1679\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "processed_datasets"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cbe750cf",
   "metadata": {},
   "source": [
    "## 加载本地数据集"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f6492356",
   "metadata": {},
   "source": [
    "### 直接加载文件为数据集"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "d05c9909",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 强制将整个数据集视为 训练集，直接返回 Dataset 对象而非 DatasetDict\n",
    "# 不加split=\"train\"的话，会返回DatasetDict\n",
    "dataset=load_dataset(\"csv\",data_files=\"ChineseNlpCorpus-master\\ChnSentiCorp_htl_all.csv\",split=\"train\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ead0b7f0",
   "metadata": {},
   "source": [
    "### 直接加载文件夹内全部文件作为数据集"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "86bea1f8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(23298, 2)"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#个数是原来csv数量的三倍，原来的七千多，这个两万两千多，全是训练集\n",
    "dataset=load_dataset(\"csv\",data_dir=\"./tst_fold\",split=\"train\")\n",
    "dataset.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "34f3ddda",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['label', 'review'],\n",
       "        num_rows: 23298\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset=load_dataset(\"csv\",data_dir=\"./tst_fold\")#不强制train的话，会变成DatasetDict类型对象\n",
    "dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2f0bbfb7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "657ff7bea1f04ef7bd9d1f1ac1263bf4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating train split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "(15532, 2)"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\"同时加载两个文件为训练集\"\"\"\n",
    "dataset=load_dataset(\"csv\",data_files=[\"./tst_fold/ChnSentiCorp_htl_all copy.csv\",\"./tst_fold/ChnSentiCorp_htl_all copy 2.csv\"],split=\"train\")\n",
    "dataset.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5724985d",
   "metadata": {},
   "source": [
    "### 通过预先加载的其他格式转换加载数据集"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "id": "23d96d60",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "data=pd.read_csv(\"tst_fold\\ChnSentiCorp_htl_all copy 2.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "id": "a51459f6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>review</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>距离川沙公路较近,但是公交指示不对,如果是\"蔡陆线\"的话,会非常麻烦.建议用别的路线.房间较...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>商务大床房，房间很大，床有2M宽，整体感觉经济实惠不错!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>早餐太差，无论去多少人，那边也不加食品的。酒店应该重视一下这个问题了。房间本身很好。</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>宾馆在小街道上，不大好找，但还好北京热心同胞很多~宾馆设施跟介绍的差不多，房间很小，确实挺小...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   label                                             review\n",
       "0      1  距离川沙公路较近,但是公交指示不对,如果是\"蔡陆线\"的话,会非常麻烦.建议用别的路线.房间较...\n",
       "1      1                       商务大床房，房间很大，床有2M宽，整体感觉经济实惠不错!\n",
       "2      1         早餐太差，无论去多少人，那边也不加食品的。酒店应该重视一下这个问题了。房间本身很好。\n",
       "3      1  宾馆在小街道上，不大好找，但还好北京热心同胞很多~宾馆设施跟介绍的差不多，房间很小，确实挺小..."
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[:4]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "19be4112",
   "metadata": {},
   "source": [
    "使用Dataset.from_pandas直接转化pandas为Dataset格式"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "id": "92d54159",
   "metadata": {},
   "outputs": [],
   "source": [
    "# \"\"\"from_csv。。。。。都可以\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "id": "7e7f9a01",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['label', 'review'],\n",
       "    num_rows: 7766\n",
       "})"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset=Dataset.from_pandas(data)\n",
    "dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "29afe8cc",
   "metadata": {},
   "source": [
    "## Dataset with DataCollator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "id": "0514a8da",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import DataCollatorWithPadding\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "id": "c14e9fba",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset=load_dataset(\"csv\",data_files=\"tst_fold\\ChnSentiCorp_htl_all copy 2.csv\",split=\"train\")\n",
    "dataset=dataset.filter(lambda x:x[\"review\"] is not None)#筛掉空评论"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b734cab6",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "核心功能：\n",
    "分词：将原始评论文本（review字段）转换为BERT的输入格式（input_ids, attention_mask等）。\n",
    "标签映射：将label字段的值复制到labels字段，供模型训练时计算损失。\n",
    "\"\"\"\n",
    "def process_func(examples):\n",
    "    tokenized_examples=tokenizer(examples[\"review\"],max_length=128,truncation=True)\n",
    "    tokenized_examples[\"labels\"]=examples[\"label\"]\n",
    "    return tokenized_examples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "87f31122",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['input_ids', 'token_type_ids', 'attention_mask', 'labels'],\n",
       "    num_rows: 7765\n",
       "})"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenized_dataset=dataset.map(process_func,batched=True,remove_columns=dataset.column_names)\n",
    "tokenized_dataset\n",
    "# 只有以下四个字段可以用官方的collator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "id": "2914b22b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'input_ids': [[101, 6655, 4895, 2335, 3763, 1062, 6662, 6772, 6818, 117, 852, 3221, 1062, 769, 2900, 4850, 679, 2190, 117, 1963, 3362, 3221, 107, 5918, 7355, 5296, 107, 4638, 6413, 117, 833, 7478, 2382, 7937, 4172, 119, 2456, 6379, 4500, 1166, 4638, 6662, 5296, 119, 2791, 7313, 6772, 711, 5042, 1296, 119, 102], [101, 1555, 1218, 1920, 2414, 2791, 8024, 2791, 7313, 2523, 1920, 8024, 2414, 3300, 100, 2160, 8024, 3146, 860, 2697, 6230, 5307, 3845, 2141, 2669, 679, 7231, 106, 102], [101, 3193, 7623, 1922, 2345, 8024, 3187, 6389, 1343, 1914, 2208, 782, 8024, 6929, 6804, 738, 679, 1217, 7608, 1501, 4638, 511, 6983, 2421, 2418, 6421, 7028, 6228, 671, 678, 6821, 702, 7309, 7579, 749, 511, 2791, 7313, 3315, 6716, 2523, 1962, 511, 102]], 'token_type_ids': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], 'labels': [1, 1, 1]}\n"
     ]
    }
   ],
   "source": [
    "print(tokenized_dataset[:3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "id": "a9cf575f",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "每个批次填充后统一长度的核心意义是：\n",
    "\n",
    "满足硬件加速的输入要求，实现高效并行计算。\n",
    "减少资源浪费，通过按需填充优化内存和计算。\n",
    "保持模型输入一致性，确保注意力机制等模块正确工作。\n",
    "动态填充允许不同批次有不同的长度，但同一批次内必须统一，从而在灵活性和效率之间达到最佳平衡\n",
    "\"\"\"\n",
    "collator=DataCollatorWithPadding(tokenizer=tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "id": "f1aa5752",
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.utils.data import DataLoader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "id": "687ff05e",
   "metadata": {},
   "outputs": [],
   "source": [
    "dl=DataLoader(tokenized_dataset,batch_size=4,collate_fn=collator,shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "id": "a1ff2e80",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "You're using a BertTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0,\n",
       " {'input_ids': tensor([[  101,  1333,  1044,  2137,  4638,  2400,  3828,  7649,  2421,  8024,\n",
       "            852,  4895,  1215,   752,  4638,  1765,  3175,  6772,  6823,  8024,\n",
       "           2792,   809,  2523,  4310,  6499,  1765,  2137,   749,   671,   702,\n",
       "           3241,   677,  8024,  3766,  2682,  1168,  1057,   857,   809,  1400,\n",
       "           1355,  4385,  2523,  2397,  1112,  8024,   671,  4157,  6963,   679,\n",
       "           3683,  3959,  4012,  7649,  2421,   722,  5102,  4638,  2792,  6458,\n",
       "           3215,  5277,  6983,  2421,  2345,  8024,   754,  3221,  7716,   677,\n",
       "           5468,  5143,  3025,  4923,  8024,  1762,  1922,  1333,  4638,   676,\n",
       "           1921,  1059,  6963,  3121,  2137,  4040,  4510,   511,  1928,   671,\n",
       "           1921,  3241,   677,  3178,  6804,   857,  4638,  1126,   702,   782,\n",
       "           2802,   749,   671,  2162,  4638,  4277,  8024,  1427,  2533,  2523,\n",
       "           8024,   671,  3241,   677,  3766,  4717,  1962,  8024,  3193,   677,\n",
       "           1440,  6401,  1184,  1378,  8024,  3302,  1218,   102],\n",
       "         [  101,  6983,  2421,  2791,  7313,  4508,  7010,  1419,  7030,  6814,\n",
       "           7770,   117,   698,  7028,  2512,  1510,  2145,   782,  6716,   860,\n",
       "            978,  2434,   119,  2769,  1057,   857,  4638,  3221,  8430,  8144,\n",
       "            117,  1157,  2458,  7305,  2345,  4157,  6158,  4221,   948,   119,\n",
       "           5018,   753,  1921,  1726,  3341,  2458,  7305,  1086,  3613,  6158,\n",
       "           7440,  1168,   119,  2940,  2791, 12463,   117,  2658,  1105,  4924,\n",
       "           1962,   117,   852,  2414,  1296,   677,  3300,   676,  1905,  3738,\n",
       "           4157,   117,  6842,  2791,  3198,  3300,  1469,  2600,  1378,  1353,\n",
       "           2418,   117,  1962,  6496,  3766,   784,   720,  1353,  2418,   119,\n",
       "           1391,  3193,  7649,   117,   872,  1391,   691,  6205,  4638,  6862,\n",
       "           2428,  6206,  6628,   677,  3302,  1218,  1447,  3059,  4669,  2094,\n",
       "           4638,  6862,  2428,   117,  1415,  1156,  3430,  2094,   677,   833,\n",
       "            784,   720,  6963,   679,   833,  4522,   678,   102],\n",
       "         [  101,  6006,  3300,   671,  2137,  2590,  2682,  1114,  1906,  8024,\n",
       "            852,  3315,   809,   711,   833,  3683,   100,  4819,  3862,  5905,\n",
       "           1921,   100,  1962,  4157,  1450,   511,  1343,   749,  2798,  1355,\n",
       "           4385,  7368,  1079,  1282,  1146,  5774,  1117,  8024,  7567,  3300,\n",
       "            100,  2399,   719,  1927,   934,  2697,   100,   511,   684,  2791,\n",
       "           7313,  6392,  1906,  5042,  7358,  8024,  1310,  4495,  2802,  2812,\n",
       "           2533,   679,  1962,   511,  5632,  3341,  3717,  6820,  2372,  7224,\n",
       "           1456,   511,  2600,   722,  4685,  2496,   671,  1847,  5128,  3864,\n",
       "            511,   702,   782,  6371,   711,  6820,   679,  1963,   671,   763,\n",
       "           4689,  2356,  4638,  3124,  2424,  2875,  2521,  2792,   511,  1041,\n",
       "           1071,  7030,   123,  3215,  8025,  8025,  5050,  3221,  4212,  7560,\n",
       "           4638,   511,  4692,  3341,  6821,  5102,   100,   772,  3326,  2466,\n",
       "           6983,  2421,   100,  3221,   679,  5543,   857,   102],\n",
       "         [  101,  6983,  2421,   855,  5390,   948,   738,  1962,  2823,  8024,\n",
       "           2218,  1762,  2412,  3217,  6662,  1469,   704,  3777,  6662,  7770,\n",
       "           3373,  1366,   677,  8024,  5722,  2123,  4510,  1690,  2190,  7481,\n",
       "            511,  1920,  1828,   679,  1920,  8024,   679,  6814,  6929,  1921,\n",
       "            782,   679,  1914,  8024,   948,   738,  3926,  1112,   511,  1184,\n",
       "           1378,  9233,  8277,  6862,  2428,   738,  4007,  2571,   511,   857,\n",
       "            130,  3517,  3403,  2791,  8024,  3308,  6205,  8024,  7716,  6662,\n",
       "            677,  4638,  1602,  1709,   738,  3766,   784,   720,  2512,  1510,\n",
       "            511,  2791,  7313,   510,  2414,  7215,  2923,  2397,  1112,  4638,\n",
       "           8024,  1372,  3221,   702,   782,  2697,  6230,  2414,  1469,  3359,\n",
       "           1928,   974,  6763,  8024,   679,   739,  2679,   511,  6983,  2421,\n",
       "           4638,  7392,  7509,   679,  3221,  2523,  1962,  8024,  6624,  2443,\n",
       "           7027,  4638,  1898,  7509,  4924,  1920,   763,   102]]), 'token_type_ids': tensor([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "          0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "          0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "          0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "          0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "          0, 0, 0, 0, 0, 0, 0, 0],\n",
       "         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "          0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "          0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "          0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "          0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "          0, 0, 0, 0, 0, 0, 0, 0],\n",
       "         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "          0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "          0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "          0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "          0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "          0, 0, 0, 0, 0, 0, 0, 0],\n",
       "         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "          0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "          0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "          0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "          0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "          0, 0, 0, 0, 0, 0, 0, 0]]), 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "          1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "          1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "          1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "          1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "          1, 1, 1, 1, 1, 1, 1, 1],\n",
       "         [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "          1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "          1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "          1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "          1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "          1, 1, 1, 1, 1, 1, 1, 1],\n",
       "         [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "          1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "          1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "          1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "          1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "          1, 1, 1, 1, 1, 1, 1, 1],\n",
       "         [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "          1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "          1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "          1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "          1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "          1, 1, 1, 1, 1, 1, 1, 1]]), 'labels': tensor([1, 0, 0, 1])})"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "next(enumerate(dl))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ea47463e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([4, 128])\n",
      "torch.Size([4, 128])\n",
      "torch.Size([4, 128])\n",
      "torch.Size([4, 128])\n",
      "torch.Size([4, 128])\n",
      "torch.Size([4, 128])\n",
      "torch.Size([4, 89])\n",
      "torch.Size([4, 128])\n",
      "torch.Size([4, 128])\n",
      "torch.Size([4, 76])\n",
      "torch.Size([4, 128])\n"
     ]
    }
   ],
   "source": [
    "num=0\n",
    "for batch in dl:\n",
    "    print(batch[\"input_ids\"].size())#查看每一批次的最长句子填充到的长度\n",
    "    num+=1\n",
    "    if num>10:\n",
    "        break"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "39254a2a",
   "metadata": {},
   "source": [
    "## 文本分类实战"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "462948cf",
   "metadata": {},
   "source": [
    "### step1:导入相关包"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "id": "9cc62c08",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import AutoTokenizer,AutoModelForSequenceClassification\n",
    "from datasets import load_dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "790175ff",
   "metadata": {},
   "source": [
    "### step2:加载数据集"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "id": "6d2bcdd4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['label', 'review'],\n",
       "    num_rows: 7765\n",
       "})"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset=load_dataset(\"csv\",data_files=\"tst_fold\\ChnSentiCorp_htl_all copy.csv\",split=\"train\")\n",
    "dataset=dataset.filter(lambda x: x[\"review\"] is not None)\n",
    "dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d8ce2d55",
   "metadata": {},
   "source": [
    "### step3:划分数据集"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "id": "d18d9412",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['label', 'review'],\n",
       "        num_rows: 6988\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['label', 'review'],\n",
       "        num_rows: 777\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "datasets=dataset.train_test_split(test_size=0.1)\n",
    "datasets"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "783065b0",
   "metadata": {},
   "source": [
    "### step4:创建dataloader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "id": "28ad585b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6299e873f2b14f68b66cce29ce35e129",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/6988 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3011e90cac3446aab7f91ad4b4cec44d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/777 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['input_ids', 'token_type_ids', 'attention_mask', 'labels'],\n",
       "        num_rows: 6988\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['input_ids', 'token_type_ids', 'attention_mask', 'labels'],\n",
       "        num_rows: 777\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 117,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "tokenizer=AutoTokenizer.from_pretrained(r\"E:\\HuggFace_model\\rbt3\")\n",
    "def process_func(examples):\n",
    "    tokenized_examples=tokenizer(examples[\"review\"],max_length=128,truncation=True)\n",
    "    tokenized_examples[\"labels\"]=examples[\"label\"]\n",
    "    return tokenized_examples\n",
    "tokenized_datasets=datasets.map(process_func,batched=True,remove_columns=datasets[\"train\"].column_names)\n",
    "tokenized_datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "id": "9db9dc86",
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.utils.data import DataLoader\n",
    "from transformers import DataCollatorWithPadding\n",
    "\n",
    "trainset,validset=tokenized_datasets[\"train\"],tokenized_datasets[\"test\"]#训练集，测试集\n",
    "\n",
    "# 加载成 dataloader\n",
    "train_loader=DataLoader(trainset,batch_size=32,shuffle=True,collate_fn=DataCollatorWithPadding(tokenizer=tokenizer))\n",
    "valid_loader=DataLoader(validset,batch_size=64,shuffle=False,collate_fn=DataCollatorWithPadding(tokenizer=tokenizer))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "id": "b166a54a",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "You're using a BertTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0,\n",
       " {'input_ids': tensor([[ 101, 1920, 1828,  ...,    0,    0,    0],\n",
       "         [ 101, 2791, 7313,  ..., 4125, 6756,  102],\n",
       "         [ 101, 3221, 1762,  ...,    0,    0,    0],\n",
       "         ...,\n",
       "         [ 101,  122,  510,  ..., 3198, 7313,  102],\n",
       "         [ 101, 1762,  677,  ...,    0,    0,    0],\n",
       "         [ 101, 1168, 4680,  ..., 3241,  677,  102]]), 'token_type_ids': tensor([[0, 0, 0,  ..., 0, 0, 0],\n",
       "         [0, 0, 0,  ..., 0, 0, 0],\n",
       "         [0, 0, 0,  ..., 0, 0, 0],\n",
       "         ...,\n",
       "         [0, 0, 0,  ..., 0, 0, 0],\n",
       "         [0, 0, 0,  ..., 0, 0, 0],\n",
       "         [0, 0, 0,  ..., 0, 0, 0]]), 'attention_mask': tensor([[1, 1, 1,  ..., 0, 0, 0],\n",
       "         [1, 1, 1,  ..., 1, 1, 1],\n",
       "         [1, 1, 1,  ..., 0, 0, 0],\n",
       "         ...,\n",
       "         [1, 1, 1,  ..., 1, 1, 1],\n",
       "         [1, 1, 1,  ..., 0, 0, 0],\n",
       "         [1, 1, 1,  ..., 1, 1, 1]]), 'labels': tensor([1, 0, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 0, 0, 1, 1, 1, 0, 1, 1, 1,\n",
       "         0, 1, 0, 1, 1, 1, 1, 0])})"
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "next(enumerate(train_loader))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7278bca6",
   "metadata": {},
   "source": [
    "### step5:创建模型和优化器"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "id": "bfa57bdb",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at E:\\HuggFace_model\\rbt3 were not used when initializing BertForSequenceClassification: ['cls.predictions.transform.dense.weight', 'cls.predictions.transform.LayerNorm.weight', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.bias', 'cls.predictions.decoder.weight', 'cls.predictions.transform.dense.bias']\n",
      "- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at E:\\HuggFace_model\\rbt3 and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "from torch.optim import Adam\n",
    "model=AutoModelForSequenceClassification.from_pretrained(r\"E:\\HuggFace_model\\rbt3\")#模型\n",
    "\n",
    "if torch.cuda.is_available():\n",
    "    model=model.cuda()\n",
    "\n",
    "\"\"\"因为做的是迁移学习，不需要那么高的学习率\"\"\"\n",
    "optimizer=Adam(model.parameters(),lr=2e-5)#优化器"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7079a143",
   "metadata": {},
   "source": [
    "### step6:训练与验证"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "id": "ff2f8189",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def evaluate():\n",
    "    model.eval()\n",
    "    acc_num=0\n",
    "    with torch.inference_mode():#禁止修改张量，禁止梯度计算\n",
    "        for batch in valid_loader:\n",
    "            if torch.cuda.is_available():\n",
    "                batch={k:v.cuda() for k,v in batch.items()}\n",
    "            output=model(**batch)\n",
    "            pred=torch.argmax(output.logits,dim=-1)\n",
    "            acc_num+=(pred.long()==batch[\"labels\"].long()).float().sum()\n",
    "    return acc_num/len(validset)\n",
    "\n",
    "def train(epoch=3,log_step=100):\n",
    "    global_step=0\n",
    "    for ep in range(epoch):\n",
    "        model.train()\n",
    "        for batch in train_loader:\n",
    "            if torch.cuda.is_available():\n",
    "                \"\"\"\n",
    "                转移后的 batch 中的每个张量（v）都在 GPU 上，而字典结构（k）本身不涉及设备，只是字段名。\n",
    "                \"\"\"\n",
    "                batch={k:v.cuda() for k,v in batch.items()}#k是键，id,labels等等，v就是值，把这些值搬到GPU\n",
    "            optimizer.zero_grad()\n",
    "            output=model(**batch)\n",
    "            output.loss.backward()#model的output有loss，来做反向传播\n",
    "            optimizer.step()\n",
    "            if global_step%100==0:\n",
    "                print(f\"{ep+1}轮，global_step:{global_step},loss:{output.loss.item()}\")\n",
    "            global_step+=1\n",
    "        acc=evaluate()\n",
    "        print(f\"ep:{ep},acc:{acc}\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "575787f6",
   "metadata": {},
   "source": [
    "### step:7训练"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "id": "6e73de1e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1轮，global_step:0,loss:0.48661738634109497\n",
      "1轮，global_step:100,loss:0.1830390840768814\n",
      "1轮，global_step:200,loss:0.07551318407058716\n",
      "ep:0,acc:0.8931788802146912\n",
      "2轮，global_step:300,loss:0.10754069685935974\n",
      "2轮，global_step:400,loss:0.3556542992591858\n",
      "ep:1,acc:0.8944659233093262\n",
      "3轮，global_step:500,loss:0.04472322016954422\n",
      "3轮，global_step:600,loss:0.23474234342575073\n",
      "ep:2,acc:0.9047619104385376\n"
     ]
    }
   ],
   "source": [
    "train()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "448a63c8",
   "metadata": {},
   "source": [
    "### step8:预测"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "id": "daff2942",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "输入:我觉得这家酒店很好吃,\n",
      "情绪判别:好评\n"
     ]
    }
   ],
   "source": [
    "sen=\"我觉得这家酒店很好吃\"\n",
    "\n",
    "id2label={0:\"差评\",1:\"好评\"}\n",
    "\n",
    "model.eval()\n",
    "with torch.inference_mode():\n",
    "    inputs=tokenizer(sen,return_tensors=\"pt\")\n",
    "    inputs={k:v.cuda() for k,v in inputs.items()}\n",
    "    logits=model(**inputs).logits\n",
    "    # print(logits.reshape(-1))\n",
    "    pred=torch.argmax(logits.reshape(-1),dim=0)\n",
    "    print(f\"输入:{sen},\\n情绪判别:{id2label.get(pred.item())}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8622ff70",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "transformers",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.21"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
